import "dotenv/config";
import { TextLoader } from "langchain/document_loaders/fs/text";
import {
  RecursiveCharacterTextSplitter,
  SupportedTextSplitterLanguages,
  TokenTextSplitter,
} from "langchain/text_splitter";
// console.log(SupportedTextSplitterLanguages);
const textLoader = new TextLoader("./data/kong.txt");
const text = await textLoader.load();
// console.log(text);

// const splitter = new RecursiveCharacterTextSplitter({
//   chunkSize: 64,
//   chunkOverlap: 16,
// });

// const splitDocs = await splitter.splitDocuments(text);
// console.log(splitDocs);

// const js = `
// function myFunction(name,job){
// 	console.log("Welcome " + name + ", the " + job);
// }

// myFunction('Harry Potter','Wizard')

// function forFunction(){
// 	for (let i=0; i<5; i++){
//         console.log("这个数字是" + i)
// 	}
// }

// forFunction()
// `;
// RecursiveCharacterTextSplitter.fromLanguage
// const splitter = RecursiveCharacterTextSplitter.fromLanguage("js", {
//   chunkSize: 64,
//   chunkOverlap: 0,
// });
// const jsOutput = await splitter.createDocuments([js]);
// console.log(jsOutput[0].metadata.loc);

// TokenTextSplitter
// const tokenSpliter = new TokenTextSplitter({
//   chunkSize: 10,
//   chunkOverlap: 0,
// });

// const token =
//   "I stand before you today the representative of a family in grief, in a country in mourning before a world in shock.";

// const tokenOutput = await tokenSpliter.createDocuments([token]);
// console.log(tokenOutput);
